{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9e200e8f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.1967],\n",
       "        [0.1589]], grad_fn=<AddmmBackward0>)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from tqdm import tqdm\n",
    "\n",
    "net = nn.Sequential(nn.Linear(4, 8), nn.ReLU(), nn.Linear(8, 1)) # 单隐藏层\n",
    "X = torch.rand(2, 4)\n",
    "net(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d1952982",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OrderedDict([('0.weight', tensor([[ 0.4847, -0.4526,  0.0317, -0.2925],\n",
      "        [ 0.2778,  0.3856,  0.4092, -0.1835],\n",
      "        [ 0.3982, -0.4746,  0.2345, -0.1104],\n",
      "        [-0.2298, -0.3835,  0.2261, -0.1941],\n",
      "        [-0.4543,  0.1339,  0.2965,  0.2892],\n",
      "        [-0.4103, -0.4889,  0.1125, -0.0478],\n",
      "        [-0.0757,  0.4223, -0.4418,  0.0792],\n",
      "        [ 0.1668, -0.1568,  0.3403, -0.0880]])), ('0.bias', tensor([-0.0674, -0.1886, -0.4037,  0.3223, -0.4523,  0.4107, -0.4542,  0.0603])), ('2.weight', tensor([[-0.0631,  0.1582, -0.2782,  0.3282, -0.2670, -0.3435,  0.2095, -0.3272]])), ('2.bias', tensor([0.1773]))])\n"
     ]
    }
   ],
   "source": [
    "print(net.state_dict())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2417e58d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'torch.nn.parameter.Parameter'>\n",
      "Parameter containing:\n",
      "tensor([0.1773], requires_grad=True)\n",
      "tensor([0.1773])\n"
     ]
    }
   ],
   "source": [
    "print(type(net[2].bias))\n",
    "print(net[2].bias)\n",
    "print(net[2].bias.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ee9d6e43",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net[2].weight.grad == None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b65508be",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('weight', torch.Size([8, 4])) ('bias', torch.Size([8]))\n",
      "('0.weight', torch.Size([8, 4])) ('0.bias', torch.Size([8])) ('2.weight', torch.Size([1, 8])) ('2.bias', torch.Size([1]))\n"
     ]
    }
   ],
   "source": [
    "print(*[(name, param.shape) for name, param in net[0].named_parameters()])\n",
    "print(*[(name, param.shape) for name, param in net.named_parameters()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ff93cf86",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.1773])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net.state_dict()['2.bias'].data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b085bab1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 4/4 [00:00<00:00, 4260.34it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[-0.2900],\n",
       "        [-0.2900]], grad_fn=<AddmmBackward0>)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def block1():\n",
    "    return nn.Sequential(nn.Linear(4, 8), nn.ReLU(), nn.Linear(8, 4), nn.ReLU())\n",
    "\n",
    "def block2():\n",
    "    net = nn.Sequential()\n",
    "    for i in tqdm(range(4)):\n",
    "        net.add_module(f'block {i}', block1())\n",
    "    return net\n",
    "\n",
    "rgnet = nn.Sequential(block2(), nn.Linear(4, 1))\n",
    "rgnet(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "04d99902",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sequential(\n",
      "  (0): Sequential(\n",
      "    (block 0): Sequential(\n",
      "      (0): Linear(in_features=4, out_features=8, bias=True)\n",
      "      (1): ReLU()\n",
      "      (2): Linear(in_features=8, out_features=4, bias=True)\n",
      "      (3): ReLU()\n",
      "    )\n",
      "    (block 1): Sequential(\n",
      "      (0): Linear(in_features=4, out_features=8, bias=True)\n",
      "      (1): ReLU()\n",
      "      (2): Linear(in_features=8, out_features=4, bias=True)\n",
      "      (3): ReLU()\n",
      "    )\n",
      "    (block 2): Sequential(\n",
      "      (0): Linear(in_features=4, out_features=8, bias=True)\n",
      "      (1): ReLU()\n",
      "      (2): Linear(in_features=8, out_features=4, bias=True)\n",
      "      (3): ReLU()\n",
      "    )\n",
      "    (block 3): Sequential(\n",
      "      (0): Linear(in_features=4, out_features=8, bias=True)\n",
      "      (1): ReLU()\n",
      "      (2): Linear(in_features=8, out_features=4, bias=True)\n",
      "      (3): ReLU()\n",
      "    )\n",
      "  )\n",
      "  (1): Linear(in_features=4, out_features=1, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(rgnet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d9161c90",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([ 0.0074,  0.0063, -0.0079,  0.0104]), tensor(0.))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def init_params(m):\n",
    "    if type(m) == nn.Linear:\n",
    "        nn.init.normal_(m.weight, 0, 0.01)\n",
    "        nn.init.zeros_(m.bias)\n",
    "        \n",
    "net.apply(init_params)\n",
    "net[0].weight.data[0], net[0].bias.data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "0420fbbb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([1., 1., 1., 1.]), tensor(0.))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def init_constant(m):\n",
    "    if isinstance(m, nn.Linear):\n",
    "        nn.init.constant_(m.weight, 1)\n",
    "        nn.init.zeros_(m.bias)\n",
    "        \n",
    "net.apply(init_constant)\n",
    "net[0].weight.data[0], net[0].bias.data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2aac526b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([ 0.0009, -0.6261, -0.5045, -0.6563])\n",
      "tensor([42., 42., 42., 42., 42., 42., 42., 42.])\n"
     ]
    }
   ],
   "source": [
    "def init_xavier(m):\n",
    "    if type(m) == nn.Linear:\n",
    "        nn.init.xavier_uniform_(m.weight)\n",
    "def init_42(m):\n",
    "    if type(m) == nn.Linear:\n",
    "        nn.init.constant_(m.weight, 42)\n",
    "\n",
    "net[0].apply(init_xavier)\n",
    "net[2].apply(init_42)\n",
    "print(net[0].weight.data[0])\n",
    "print(net[2].weight.data[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "04842478",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Init weight torch.Size([8, 4])\n",
      "Init weight torch.Size([1, 8])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0000, -0.0000, -0.0000,  0.0000],\n",
       "        [-9.6351, -0.0000, -7.5010,  9.3118]], grad_fn=<SliceBackward0>)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def my_init(m):\n",
    "    if isinstance(m, nn.Linear):\n",
    "        print(\n",
    "            \"Init\",\n",
    "            *[(name, param.shape) for name, param in m.named_parameters()][0]\n",
    "        )\n",
    "        nn.init.uniform_(m.weight, -10, 10)\n",
    "        m.weight.data *= m.weight.data.abs() >= 5 # 将小于5的元素置0\n",
    "\n",
    "net.apply(my_init)\n",
    "net[0].weight[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "41215ca8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([42.,  1.,  1.,  1.])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net[0].weight.data[:] += 1\n",
    "net[0].weight.data[0, 0] = 42\n",
    "net[0].weight.data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "2fba20b9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([True, True, True, True, True, True, True, True])\n",
      "tensor([True, True, True, True, True, True, True, True])\n"
     ]
    }
   ],
   "source": [
    "shared = nn.Linear(8, 8)\n",
    "net = nn.Sequential(\n",
    "    nn.Linear(4, 8), \n",
    "    nn.ReLU(),\n",
    "    shared,\n",
    "    nn.ReLU(),\n",
    "    shared,\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(8, 1)\n",
    ")\n",
    "net(X)\n",
    "print(net[2].weight.data[0] == net[4].weight.data[0])\n",
    "net[2].weight.data[0, 0] = 100\n",
    "print(net[2].weight.data[0] == net[4].weight.data[0])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "d2l",
   "language": "python",
   "name": "d2l"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.17"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
